{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# 配置卷积神经网络的架构参数\n",
    "INPUT_NODE = 784\n",
    "OUTPUT_NODE = 10\n",
    "\n",
    "IMAGE_SIZE = 28\n",
    "NUM_CHANNELS = 1\n",
    "NUM_LABELS = 10\n",
    "\n",
    "# 第一层卷积层的尺寸和深度\n",
    "CONV1_DEEP = 32\n",
    "CONV1_SIZE = 5\n",
    "# 第二层卷积层的尺寸和深度\n",
    "CONV2_DEEP = 64\n",
    "CONV2_SIZE = 5\n",
    "# 全连接层的结点个数\n",
    "FC_SIZE = 512\n",
    "\n",
    "\n",
    "# 定义卷积神经网络的前向传播过程。这里添加了一个新的参数train，用于区分训练过程和测试过程。在这个程序中将用到dropout方法，\n",
    "# dropout方法可进一步提升模型的可靠性并防止过拟合，dropout过程只在训练时使用\n",
    "def inference(input_tensor, train, regularizer):\n",
    "    # 声明第一层卷积层的变量并实现前向传播过程。通过使用不同命名空间来隔离不同层的变量，让每一层中的变量命名只需要考虑在当前层的作用，\n",
    "    # 不需担心重命名的问题。第一层输出为28×28×32的张量\n",
    "    with tf.variable_scope('layer1-conv1'):\n",
    "        conv1_weights = tf.get_variable('weight', [CONV1_SIZE, CONV1_SIZE, NUM_CHANNELS, CONV1_DEEP],\n",
    "                                        initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        conv1_biases = tf.get_variable('bias', [CONV1_DEEP], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "        # 使用边长为5，深度为32的卷积核，卷积核的移动步幅为1，且使用0填充\n",
    "        conv1 = tf.nn.conv2d(input_tensor, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "        relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))\n",
    "\n",
    "    # 实现第二层池化层的前向传播过程。该最大池化层卷积核边长为2，使用0填充，移动步幅为2.\n",
    "    # 该层的输入为28×28×32的张量，输出为14×14×32的张量\n",
    "    with tf.name_scope('layer2-pool1'):\n",
    "        pool1 = tf.nn.max_pool(relu1, ksize=[1, 2, 2, 1],strides=[1,1,1,1], padding='SAME')\n",
    "\n",
    "    # 声明第三层卷积层的变量并实现前向传播过程，该卷积层的输入为14×14×32的张量，输出为14×14×64的矩阵\n",
    "    with tf.variable_scope('layer3-conv2'):\n",
    "        conv2_weights = tf.get_variable('weight', [CONV2_SIZE, CONV2_SIZE, CONV1_DEEP, CONV2_DEEP],\n",
    "                                        initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        conv2_biases = tf.get_variable('bias', [CONV2_DEEP], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "        # 使用尺寸为5×5，深度为64的卷积核，卷积核的移动步幅为1，且使用0填充\n",
    "        conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "        relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))\n",
    "\n",
    "    # 实现第四层池化层的前向传播过程，输入为14×14×64，输出为7×7×64的张量\n",
    "    with tf.name_scope('layer4-pool2'):\n",
    "        pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "    # 将第四层池化层的输出转化为第五层全连接层的输入格式。第四层为7×7×64的张量，第五层输入为向量，所以需要将该张量拉成一个向量\n",
    "    # pool2.get_shape函数取第四层输出张量的维度，每层的输入输出都为一个BATCH的张量，所以这里得到的维度也包含一个BATCH中数据的数量。\n",
    "    pool_shape = pool2.get_shape().as_list()\n",
    "\n",
    "    # 计算将张量拉直成向量后的长度，该长度等于张量维度累乘。注意这里的pool_shape[0]为一个batch中数据的个数\n",
    "    nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]\n",
    "\n",
    "    # 通过tf.reshape函数将第四层的输出变成一个batch的向量\n",
    "    reshaped = tf.reshape(pool2, [pool_shape[0], nodes])\n",
    "\n",
    "    # 声明第五层全连接层的变量并实现前向传播过程。输入长度为3136的向量，输出长度为512的向量。该层引入了dropout的概念，\n",
    "    # dropout在训练时随机将部分结点的输出改为0.dropout一般只在全连接层而不是卷积层或池化层使用。\n",
    "    with tf.variable_scope('layer5-fcl'):\n",
    "        fc1_weights = tf.get_variable('weight', [nodes, FC_SIZE],\n",
    "                                      initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "\n",
    "        # 只有全连接层权重需要加入正则化\n",
    "        if regularizer != None:\n",
    "            tf.add_to_collection('losses', regularizer(fc1_weights))\n",
    "        fc1_biases = tf.get_variable('bias', [FC_SIZE], initializer=tf.constant_initializer(0.1))\n",
    "\n",
    "        fc1 = tf.nn.relu(tf.matmul(reshaped, fc1_weights) + fc1_biases)\n",
    "        if train: fc1 = tf.nn.dropout(fc1, 0.5)\n",
    "\n",
    "    # 声明第六层全连接层变量并实现前向传播，输入长度为512的向量，输出长度为10的向量。输出通过softmax之后可得到最后的分类结果。\n",
    "    with tf.variable_scope('layer6-fc2'):\n",
    "        fc2_weights = tf.get_variable('weight', [FC_SIZE, NUM_LABELS],\n",
    "                                      initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None:\n",
    "            tf.add_to_collection('losses', regularizer(fc1_weights))\n",
    "\n",
    "        fc2_biases = tf.get_variable('bias', [NUM_LABELS], initializer=tf.constant_initializer(0.1))\n",
    "        logit = tf.matmul(fc1, fc2_weights) + fc2_biases\n",
    "    return logit\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/MNIST/t10k-labels-idx1-ubyte.gz\n",
      "After 1 training steps, loss on training batch is 15.0016.\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import os\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import numpy as np\n",
    "\n",
    "# 配置神经网络的参数\n",
    "BATCH_SIZE = 8\n",
    "LEARNING_RATE_BASE = 0.8\n",
    "LEARNING_RATE_DECAY = 0.99\n",
    "REGULARIZATION_RATE = 0.0001\n",
    "TRAINING_STEPS = 10000\n",
    "MOVING_AVERAGE_DECAY = 0.99\n",
    "MODEL_SAVE_PATH = \"./model/fcn_mnist\"\n",
    "MODEL_NAME = \"fcn_mnist.ckpt\"\n",
    "\n",
    "\n",
    "def train(mnist):\n",
    "    x = tf.placeholder(tf.float32, [BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS], name='x-input')\n",
    "    y = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-output')\n",
    "\n",
    "    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)\n",
    "    # 调用推断过程\n",
    "    y_hat = inference(x, True, regularizer)\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "\n",
    "    # 定义损失函数、学习率、滑动平均操作及训练过程\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)\n",
    "    variables_average_op = variable_averages.apply(tf.trainable_variables())\n",
    "    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y_hat, labels=tf.argmax(y, 1))\n",
    "    cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
    "    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))\n",
    "    learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE,\n",
    "                                              LEARNING_RATE_DECAY)\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "\n",
    "    with tf.control_dependencies([train_step, variables_average_op]):\n",
    "        train_op = tf.no_op(name='train')\n",
    "\n",
    "    # 初始化TF持久化类\n",
    "    saver = tf.train.Saver()\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        # 在训练过程中不再测试模型在验证数据上的表现，验证和测试的过程会有独立的过程完成\n",
    "        for i in range(TRAINING_STEPS):\n",
    "            xs, ys = mnist.train.next_batch(BATCH_SIZE)\n",
    "            reshaped_xs=np.reshape(xs,(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))\n",
    "            _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y: ys})\n",
    "\n",
    "            # 每1000次迭代保存一次模型\n",
    "            if i % 1000 == 0:\n",
    "                # 输出模型在当前训练批量下的损失函数大小\n",
    "                print('After %d training steps, loss on training batch is %g.' % (step, loss_value))\n",
    "\n",
    "                # 保存当前模型，并使用global_step 参数特定地命名\n",
    "                saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)\n",
    "\n",
    "\n",
    "def main(argv=None):\n",
    "    mnist = input_data.read_data_sets('./data/MNIST/', one_hot=True)\n",
    "    train(mnist)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    tf.app.run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
